% Feature Selection By Forward Selection
clc;
clearvars;
rng('shuffle');
K=68;
DataFileName=['Data\BcwrdEliminData_ByMAE_UFS_NoB_Fit_',num2str(K),'NN_Run2.mat'];
FoldNumber=10;
[Train,Test]=GetKFoldRegressorDataFromRepository(FoldNumber,true);
EliminatedFeatures=[];
% Train.Features(:,EliminatedFeatures,:)=[];
% Train.FeaturesName(EliminatedFeatures)=[];
% Test.Features(:,EliminatedFeatures,:)=[];
% Test.FeaturesName(EliminatedFeatures)=[];
ReplaceChaseFlag=true;
if exist(DataFileName,'file')
    Yes_NO = questdlg({'Cash File Exist','Replace it?'});
    if strcmp(Yes_NO,'No') || strcmp(Yes_NO,'Cancel') || isempty(Yes_NO)
        ReplaceChaseFlag=false;
    end
end
if ReplaceChaseFlag
AllFeaturesIndex={[2:5,9,11,28:37,39] % ALL
                  [2:5,9,11,28,29]};% No Fitting
FeaturesIndex=AllFeaturesIndex{2};% No Fitting 
   
FNum=length(FeaturesIndex);
Features=cell(1,FNum);
Erorrs=zeros(FoldNumber,Test.SNum);
Features{1}=FeaturesIndex;
BestMAE=zeros(1,FNum);
AnalizeErorrs=zeros(FNum,Test.SNum*Test.SetNum);
for J=1:FNum
    MAE=zeros(FNum-J+1,1);
    for j=1:FNum-J+1
        fprintf('\nAnalyze Eliminate Feature Number % 2d(% 2d) For % 2d Features.',Features{J}(j),j,FNum-J)
        FeaturesTemp=Features{J}([1:j-1,j+1:end]);
        for f=1:FoldNumber
        KNNRegressor = KNNReg(Train.Features(:,FeaturesTemp,f),Train.Label(:,f),'NumNeighbors',K,'RegresiorMethod','InverseDistance');
        PredictedValue=GetRegValue(KNNRegressor,Test.Features(:,FeaturesTemp,f));
        Erorrs(f,:)=PredictedValue-Test.Label(:,f);
        fprintf('\nAnalyze Eliminate Feature Number % 2d(% 2d) For % 2d Features.% 3d%%',Features{J}(j),j,FNum-J,f/FoldNumber*100)
        end
        MAE(j)=mean2(abs(Erorrs));
    end
    [BestMAE(J),Best]=min(MAE);
    Features{J}=Features{J}([1:Best-1,Best+1:end]);
    Features{J+1}=Features{J};
    for f=1:FoldNumber
        KNNRegressor = KNNReg(Train.Features(:,Features{J},f),Train.Label(:,f),'NumNeighbors',K,'RegresiorMethod','InverseDistance');
        PredictedValue=GetRegValue(KNNRegressor,Test.Features(:,Features{J},f));
        Erorrs(f,:)=PredictedValue-Test.Label(:,f);
    end
    AnalizeErorrs(J,:)=reshape(Erorrs,[],1);
end
%% All Features MAE 
for f=1:FoldNumber
KNNRegressor = KNNReg(Train.Features(:,FeaturesIndex,f),Train.Label(:,f),'NumNeighbors',K,'RegresiorMethod','InverseDistance');
PredictedValue=GetRegValue(KNNRegressor,Test.Features(:,FeaturesIndex,f));
Erorrs(f,:)=PredictedValue-Test.Label(:,f);
end
MAEALL=mean2(abs(Erorrs));
%% 
FeaturesName=Train.FeaturesName;
save(DataFileName,'Features','BestMAE','AnalizeErorrs','MAEALL','FNum','FNum','EliminatedFeatures','FeaturesName')
else
load(DataFileName)
end
%%
BasePlotFileName='Plots\FrwdSel_USF_NoB_Fit_Fig';
TextSize=14;
SavePlots=0;
Selected=zeros(FNum);
for J=1:FNum
    Selected(J,Features{J})=1;
end
SelectedRank=sum(Selected,1);
[Sorted,SortIndex]=sort(SelectedRank,'descend');
SortIndex(FNum+1:end)=[];


figure(2);
set(gca,'FontSize',TextSize)
SortedFeaturesName=Test.FeaturesName(SortIndex);
stairs(1:FNum+2,[BestMAE,MAEALL,MAEALL],'linewidth',2)
for i=1:FNum
    text(i+0.5,BestMAE(i)+0.0005,SortedFeaturesName{i},'Rotation',90,'FontSize',12)
end
text(FNum+1.5,MAEALL+0.0005,'All','Rotation',90,'FontSize',10)
% ylim([0.0245,0.0435])
% title({'Mean Square Error of extracted ''w''' ,'By Forward Selection (All Features Included)'})
xlabel('Number of Features');
xlim([1,FNum+2])
ylabel('Mean Square Error');
set(gcf,'Position',[50,50,900,300]);

if SavePlots
FileName=[BasePlotFileName,'1_MAE'];
saveas(gcf,[FileName,'.jpg']);
end
%%
FeaturesIndex=1:39;
FeaturesIndex(EliminatedFeatures)=[];
SelectedFeatures=FeaturesIndex(SortIndex(1:10));
fprintf('SelectedFeatures = [%d',SelectedFeatures(1))
fprintf(',%d',SelectedFeatures(2:end))
fprintf(']\n')